{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance Evaluation of MT\n",
    "### BLEU Score: Adequacy and Fluency of Translations\n",
    "\n",
    "1. Calculate the modified n-gram precision for the full corpus for all n=1...N\n",
    "2. Calculate the geometric mean (gm-precision) of all the precisions\n",
    "3. Calculate the brevity penalty (bp) for the full corpus\n",
    "3. Calculate the BLEU by bp * gm-precision\n",
    "\n",
    "Example Calculation:\n",
    "\n",
    "* Candidate1: the the the the the the the\n",
    "* Candidate2: the cat is on the mat\n",
    "* Ref1: the cat sat on the mat\n",
    "* Ref2: there is a cat on the mat\n",
    "\n",
    "#### Modified 1-gram Precision (Measures adequacy)\n",
    "* Candidate1: $\\frac{2}{7}$\n",
    "* Candidate2: $\\frac{5}{5}$\n",
    "\n",
    "#### Modified 2-gram Precision (Measures fluency)\n",
    "* Candidate1: $\\frac{0}{1}$\n",
    "* Candidate2: $\\frac{3}{5}$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating modified 1-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the the the\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  the a cat\n",
      "Calculating modified 2-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the the the\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  the a cat\n",
      "Calculating modified 3-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the the the\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  the a cat\n",
      "\n",
      "BLEU-3:  8.568589920310384e-35\n",
      "\n",
      "Calculating modified 1-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the dog on the mat\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  there is cat on the mat\n",
      "Calculating modified 2-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the dog on the mat\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  there is cat on the mat\n",
      "Calculating modified 3-gram precision\n",
      "\tReference sentence:  the cat sat on the mat\n",
      "\t  Candidate sentence:  the dog on the mat\n",
      "\tReference sentence:  there is a cat on the mat\n",
      "\t  Candidate sentence:  there is cat on the mat\n",
      "\n",
      "BLEU-3:  0.5319658954895262\n"
     ]
    }
   ],
   "source": [
    "def unique_n_gram_string(n_gram):\n",
    "    string = ''\n",
    "    for g in n_gram[:-1]:\n",
    "        string += str(g)+'-'\n",
    "        \n",
    "    string += str(n_gram[-1])\n",
    "    return string\n",
    "\n",
    "def calculate_mod_n_gram_precision(n_gram, refs, cands):\n",
    "    \n",
    "    denominator = 0.0\n",
    "    \n",
    "    tot_bleu = 0.0\n",
    "    \n",
    "    tot_ref_length, tot_cand_length = 0, 0\n",
    "    for ref, cand in zip(refs, cands):\n",
    "        \n",
    "        print('\\tReference sentence: ',' '.join([reverse_test_dict[r] for r in ref]))\n",
    "        print('\\t  Candidate sentence: ',' '.join([reverse_test_dict[c] for c in cand]))\n",
    "\n",
    "        denominator += max(cand.size + 1 - n_gram,1)\n",
    "        tot_ref_length += ref.size\n",
    "        tot_cand_length += cand.size\n",
    "        \n",
    "        # find unique n-grams in predicted\n",
    "        cand_n_grams = [unique_n_gram_string(cand[w_i:w_i+n_gram]) for w_i in range(cand.size + 1 - n_gram)]\n",
    "        cand_n_grams = list(set(cand_n_grams))\n",
    "\n",
    "        occurences_for_unique_grams = dict(zip(cand_n_grams,[0 for _ in cand_n_grams]))\n",
    "\n",
    "        ref_n_grams = [unique_n_gram_string(ref[w_i:w_i+n_gram]) for w_i in range(ref.size + 1 - n_gram)]\n",
    "        ref_counts = Counter(ref_n_grams)\n",
    "\n",
    "        # iterates through every n_gram in the predicted\n",
    "        for w_i in range(cand.size + 1 - n_gram):        \n",
    "            c_gram = cand[w_i:w_i+n_gram]\n",
    "            gram_string = unique_n_gram_string(c_gram)\n",
    "            \n",
    "            for ref_i in range(ref.size + 1 - n_gram):\n",
    "\n",
    "                r_gram = ref[ref_i:ref_i+n_gram]\n",
    "\n",
    "                found_gram_in_actual = int(np.prod(c_gram == r_gram))\n",
    "\n",
    "                occurences_for_unique_grams[gram_string] += found_gram_in_actual\n",
    "\n",
    "        \n",
    "        for g, occ in occurences_for_unique_grams.items():\n",
    "            g_bleu = float(occ)\n",
    "            if g in ref_counts:\n",
    "                g_bleu = min(g_bleu,ref_counts[g])\n",
    "\n",
    "            tot_bleu += g_bleu\n",
    "\n",
    "    mod_n_prec = tot_bleu/denominator\n",
    "    \n",
    "    \n",
    "    return mod_n_prec, tot_ref_length, tot_cand_length\n",
    "\n",
    "\n",
    "def calculate_bleu(refs, cands, high_n):\n",
    "    weight = 1.0/high_n # using the same weight for all mod n_gram precisions\n",
    "    \n",
    "    tot_precision = []\n",
    "    for n in range(1,high_n+1): \n",
    "        print('Calculating modified %d-gram precision'%n)\n",
    "        prec, tot_ref_length, tot_cand_length = calculate_mod_n_gram_precision(n,refs,cands)\n",
    "        tot_precision.append(weight*np.log(prec + 1e-100))\n",
    "        \n",
    "    brevity_penalty = 1.0\n",
    "    \n",
    "    if tot_cand_length <= tot_ref_length:\n",
    "        brevity_penalty = np.exp(1.0-(tot_ref_length*1.0/max(tot_cand_length,1)))\n",
    "        \n",
    "    bleu = brevity_penalty * np.exp(np.sum(tot_precision))\n",
    "\n",
    "    return bleu\n",
    "\n",
    "test_dict = {'the':10,'cat':11,'sat':12,'on':13,'mat':14,'is':15,'there':16,'a':17,'dog':18}\n",
    "reverse_test_dict = dict(zip(test_dict.values(),test_dict.keys()))\n",
    "\n",
    "sample_text_refs = [['the','cat','sat','on','the','mat'],['there','is','a','cat','on','the','mat']]\n",
    "sample_refs = []\n",
    "for r in sample_text_refs:\n",
    "    sample_refs.append(np.asarray([test_dict[w] for w in r],dtype=np.int32))\n",
    "\n",
    "sample_text_cands_1 = [['the','the','the'],['the','a','cat']]\n",
    "sample_cands_1 = []\n",
    "for c1 in sample_text_cands_1:\n",
    "    sample_cands_1.append(np.asarray([test_dict[w] for w in c1],dtype=np.int32))\n",
    "\n",
    "\n",
    "sample_text_cands_2 = [['the','dog','on','the','mat'],['there','is','cat','on','the','mat']]\n",
    "sample_cands_2 = []\n",
    "for c2 in sample_text_cands_2:\n",
    "    sample_cands_2.append(np.asarray([test_dict[w] for w in c2],dtype=np.int32))\n",
    "\n",
    "\n",
    "b1 = calculate_bleu(sample_refs,sample_cands_1,3)\n",
    "print('\\nBLEU-3: ',b1)\n",
    "print()\n",
    "\n",
    "b2 = calculate_bleu(sample_refs,sample_cands_2,3)\n",
    "print('\\nBLEU-3: ',b2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
